from __future__ import division
import random
import math
import os
import sys
from sets import Set

# key: externalTrecID
# value: terms evaluated for this document
documentWithTermsDict = {}

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_06_09.smallTest.Combine.train_tfqAdded_labelsAdded_sortedByTerm"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/documents_needed_to_be_parsed_for_posting_rank_in_doc"
outputFileHanlder = open(outputFileName,"w")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    externalTrecID = lineElements[2]
    internalDocID = lineElements[3]
    term = lineElements[4]
    if externalTrecID not in documentWithTermsDict:
        documentWithTermsDict[externalTrecID] = []
        documentWithTermsDict[externalTrecID].append(term)
    else:
        if term not in documentWithTermsDict[externalTrecID]:
            documentWithTermsDict[externalTrecID].append(term)
        else:
            # do NOTHING
            pass
    

relatedDocumentsList = documentWithTermsDict.keys()
relatedDocumentsList.sort(cmp=None, key=None, reverse=False)

for trecID in relatedDocumentsList:
    outputLine = trecID + " "
    outputLine += str( len(documentWithTermsDict[trecID]) ) + " " #add the total number of terms in it
    for term in documentWithTermsDict[trecID]:
        outputLine += term + " "
    outputLine = outputLine.strip()
    outputLine += "\n"
    outputFileHanlder.write(outputLine)

inputFileHandler.close()
outputFileHanlder.close()





